#include <iostream>
#include "Belief.h"
#include "ApproximateBeliefUpdate.h"
#include "ExactBeliefUpdate.h"
#include "tiger/TigerSimulator.h"
#include "tiger/TigerSuperState.h"
#include "tiger/TigerReward.h"
#include "OnlineBAPOMDP.h"
#include <stdlib.h>
#include <time.h>

typedef TigerSimulator simulator_t;
typedef TigerSuperState super_state_t;
typedef super_state_t::action_t action_t;
typedef super_state_t::state_t state_t;
typedef super_state_t::obs_t obs_t;
typedef Belief<super_state_t> belief_t;

typedef ExactBeliefUpdate<belief_t> u_belief_t;
//typedef ApproximateBeliefUpdate<u_belief_t> app_u_belief_t;
typedef u_belief_t app_u_belief_t;
typedef OnlineBAPOMDP<app_u_belief_t, TigerReward> rl_t;

using namespace std;
int main(int argc, char *argv[]) {
    srand ( /*time(NULL)*/ 12 );
    //cout << "Hello!" << endl;
    simulator_t sim;
    rl_t rl;
    app_u_belief_t tau;
    //Initial belief
    belief_t b;
    super_state_t s1(state_t::tiger_left, 5, 3, 3, 5);
    super_state_t s2(state_t::tiger_right, 5, 3, 3, 5);
    b.set(s1, 0.5);
    b.set(s2, 0.5);
    
    //Parameters
    int d=2;
    int K=2;
    int it = 0;
    double total_r = 0;
    for(int t = 0; t < 100; t++) {  
        cout << "=================== Iteration: " << it++ << " ===================\n";        
        cout << "Belief:\n" << b << "\n";
        action_t a;
        double q = rl(b,d,d,K,a); //Take the best next action and return the expect reward
        total_r += q;
        //a = (int)TigerAction::listen;
        //double q=0;
        //Output actions
        if (a.getValue() == (int)TigerAction::open_left)
            cout << "Open Left " << ", Expected long term reward: " << q << "\n";
        else if (a.getValue() == (int)TigerAction::open_right)
            cout << "Open Right" << ", Expected long term reward: " << q << "\n";
        else
            cout << "Listen" << ", Expected long term reward: " << q << "\n";
        double reward = sim.control(a); //The reward function is known so this is useless
        cout << "Reward Received: " << reward << "\n";
        
        obs_t z = sim.sense();
        if (z.getValue() == (int)TigerObservation::hear_left) {
            cout << "Hear Left" << "\n";}
        else
            cout << "Hear Right" << "\n";
        
        if(z.isFinal()) {//If the final state is reach
            cout << "Terminated" << "\n";
            //break;
        }
        b = tau(b,a,z,K);
    }
    cout << total_r;
    return 0;
}
